{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3b1206ca",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:04.087900Z",
     "iopub.status.busy": "2023-02-07T15:04:04.087412Z",
     "iopub.status.idle": "2023-02-07T15:04:04.098135Z",
     "shell.execute_reply": "2023-02-07T15:04:04.097103Z"
    },
    "papermill": {
     "duration": 0.020992,
     "end_time": "2023-02-07T15:04:04.100749",
     "exception": false,
     "start_time": "2023-02-07T15:04:04.079757",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "input_dir = '../input/'\n",
    "working_dir = '../working/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4ee935d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:04.112373Z",
     "iopub.status.busy": "2023-02-07T15:04:04.111903Z",
     "iopub.status.idle": "2023-02-07T15:04:04.119246Z",
     "shell.execute_reply": "2023-02-07T15:04:04.117965Z"
    },
    "papermill": {
     "duration": 0.016242,
     "end_time": "2023-02-07T15:04:04.121940",
     "exception": false,
     "start_time": "2023-02-07T15:04:04.105698",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "889dc745",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:04.133246Z",
     "iopub.status.busy": "2023-02-07T15:04:04.132806Z",
     "iopub.status.idle": "2023-02-07T15:04:04.809361Z",
     "shell.execute_reply": "2023-02-07T15:04:04.807741Z"
    },
    "papermill": {
     "duration": 0.686498,
     "end_time": "2023-02-07T15:04:04.813391",
     "exception": false,
     "start_time": "2023-02-07T15:04:04.126893",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(os.path.join(input_dir, 'costa-rican-household-poverty-prediction/train.csv'))\n",
    "test = pd.read_csv(os.path.join(input_dir, 'costa-rican-household-poverty-prediction/test.csv'))\n",
    "\n",
    "train.index = train['Id'].values\n",
    "test.index = test['Id'].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cc7f190a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:04.826960Z",
     "iopub.status.busy": "2023-02-07T15:04:04.826419Z",
     "iopub.status.idle": "2023-02-07T15:04:04.846616Z",
     "shell.execute_reply": "2023-02-07T15:04:04.844944Z"
    },
    "papermill": {
     "duration": 0.029927,
     "end_time": "2023-02-07T15:04:04.850015",
     "exception": false,
     "start_time": "2023-02-07T15:04:04.820088",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def data_cleaning(data):\n",
    "    data['dependency']=np.sqrt(data['SQBdependency'])\n",
    "    data['rez_esc']=data['rez_esc'].fillna(0)\n",
    "    data['v18q1']=data['v18q1'].fillna(0)\n",
    "    data['v2a1']=data['v2a1'].fillna(0)\n",
    "    \n",
    "    conditions = [\n",
    "    (data['edjefe']=='no') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='yes') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='no') & (data['edjefa']=='yes'), \n",
    "    (data['edjefe']!='no') & (data['edjefe']!='yes') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='no') & (data['edjefa']!='no') \n",
    "    ]\n",
    "    choices = [0, 1, 1, data['edjefe'], data['edjefa']]\n",
    "    data['edjefx']=np.select(conditions, choices)\n",
    "    data['edjefx']=data['edjefx'].astype(int)\n",
    "    data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)\n",
    "    \n",
    "    meaneduc_nan=data[data['meaneduc'].isnull()][['Id','idhogar','escolari']]\n",
    "    me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()\n",
    "    for row in meaneduc_nan.iterrows():\n",
    "        idx=row[0]\n",
    "        idhogar=row[1]['idhogar']\n",
    "        m=me[me['idhogar']==idhogar]['escolari'].tolist()[0]\n",
    "        data.at[idx, 'meaneduc']=m\n",
    "        data.at[idx, 'SQBmeaned']=m*m\n",
    "        \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e15bd5ed",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:04.862435Z",
     "iopub.status.busy": "2023-02-07T15:04:04.861973Z",
     "iopub.status.idle": "2023-02-07T15:04:04.993273Z",
     "shell.execute_reply": "2023-02-07T15:04:04.991798Z"
    },
    "papermill": {
     "duration": 0.141039,
     "end_time": "2023-02-07T15:04:04.996342",
     "exception": false,
     "start_time": "2023-02-07T15:04:04.855303",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = data_cleaning(train)\n",
    "test = data_cleaning(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c5126e75",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:05.008140Z",
     "iopub.status.busy": "2023-02-07T15:04:05.006985Z",
     "iopub.status.idle": "2023-02-07T15:04:05.041493Z",
     "shell.execute_reply": "2023-02-07T15:04:05.040062Z"
    },
    "papermill": {
     "duration": 0.043404,
     "end_time": "2023-02-07T15:04:05.044417",
     "exception": false,
     "start_time": "2023-02-07T15:04:05.001013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = train.query('parentesco1==1')\n",
    "train = train.drop('parentesco1', axis=1)\n",
    "test = test.drop('parentesco1', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1f37db44",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:05.055599Z",
     "iopub.status.busy": "2023-02-07T15:04:05.055151Z",
     "iopub.status.idle": "2023-02-07T15:04:05.062838Z",
     "shell.execute_reply": "2023-02-07T15:04:05.061530Z"
    },
    "papermill": {
     "duration": 0.016415,
     "end_time": "2023-02-07T15:04:05.065542",
     "exception": false,
     "start_time": "2023-02-07T15:04:05.049127",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_numeric(data, status_name):\n",
    "    status_cols = [s for s in data.columns.tolist() if status_name in s]\n",
    "    print('status column names')\n",
    "    print(status_cols)\n",
    "    status_df = data[status_cols]\n",
    "    status_df.columns = list(range(status_df.shape[1]))\n",
    "    status_numeric = status_df.idxmax(1)\n",
    "    status_numeric.name = status_name\n",
    "    data = pd.concat([data, status_numeric], axis=1)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "daa7e21c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:05.078912Z",
     "iopub.status.busy": "2023-02-07T15:04:05.076384Z",
     "iopub.status.idle": "2023-02-07T15:04:05.409986Z",
     "shell.execute_reply": "2023-02-07T15:04:05.409062Z"
    },
    "papermill": {
     "duration": 0.342645,
     "end_time": "2023-02-07T15:04:05.413044",
     "exception": false,
     "start_time": "2023-02-07T15:04:05.070399",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "status column names\n",
      "['epared1', 'epared2', 'epared3']\n",
      "status column names\n",
      "['epared1', 'epared2', 'epared3']\n",
      "status column names\n",
      "['etecho1', 'etecho2', 'etecho3']\n",
      "status column names\n",
      "['etecho1', 'etecho2', 'etecho3']\n",
      "status column names\n",
      "['eviv1', 'eviv2', 'eviv3']\n",
      "status column names\n",
      "['eviv1', 'eviv2', 'eviv3']\n",
      "status column names\n",
      "['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']\n",
      "status column names\n",
      "['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']\n"
     ]
    }
   ],
   "source": [
    "status_name_list = ['epared', 'etecho', 'eviv', 'instlevel']\n",
    "for status_name in status_name_list:\n",
    "    train = get_numeric(train, status_name)\n",
    "    test = get_numeric(test, status_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9d59a3df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:05.430648Z",
     "iopub.status.busy": "2023-02-07T15:04:05.429779Z",
     "iopub.status.idle": "2023-02-07T15:04:05.476235Z",
     "shell.execute_reply": "2023-02-07T15:04:05.475278Z"
    },
    "papermill": {
     "duration": 0.059184,
     "end_time": "2023-02-07T15:04:05.479496",
     "exception": false,
     "start_time": "2023-02-07T15:04:05.420312",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "needless_cols = ['r4t3', 'tamhog', 'hogar_total', 'hhsize', 'v18q', 'sanitario1', 'agesq',\n",
    "                 'mobilephone', 'area1', 'female', 'epared1', 'epared2',\n",
    "                 'epared3', 'etecho1', 'etecho2', 'etecho3',\n",
    "                 'eviv1', 'eviv2', 'eviv3', 'instlevel1', 'instlevel2',\n",
    "                 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6',\n",
    "                 'instlevel7', 'instlevel8', 'instlevel9', 'abastaguafuera']\n",
    "SQB_cols = [s for s in train.columns.tolist() if 'SQB' in s]\n",
    "parentesco_cols = [s for s in train.columns.tolist() if 'parentesco' in s]\n",
    "\n",
    "needless_cols.extend(SQB_cols)\n",
    "needless_cols.extend(parentesco_cols)\n",
    "\n",
    "train = train.drop(needless_cols, axis=1)\n",
    "test = test.drop(needless_cols, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "749938f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:05.496124Z",
     "iopub.status.busy": "2023-02-07T15:04:05.495454Z",
     "iopub.status.idle": "2023-02-07T15:04:05.638981Z",
     "shell.execute_reply": "2023-02-07T15:04:05.638094Z"
    },
    "papermill": {
     "duration": 0.154864,
     "end_time": "2023-02-07T15:04:05.641659",
     "exception": false,
     "start_time": "2023-02-07T15:04:05.486795",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature columns \n",
      " 140 -> 94\n"
     ]
    }
   ],
   "source": [
    "ori_train = pd.read_csv(os.path.join(input_dir, 'costa-rican-household-poverty-prediction/train.csv'))\n",
    "ori_train_X = ori_train.drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "train_X = train.drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "print('feature columns \\n {} -> {}'.format(ori_train_X.shape[1], train_X.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f8b58181",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:05.658804Z",
     "iopub.status.busy": "2023-02-07T15:04:05.657746Z",
     "iopub.status.idle": "2023-02-07T15:04:05.697881Z",
     "shell.execute_reply": "2023-02-07T15:04:05.696959Z"
    },
    "papermill": {
     "duration": 0.051101,
     "end_time": "2023-02-07T15:04:05.700480",
     "exception": false,
     "start_time": "2023-02-07T15:04:05.649379",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_Id = train['Id']\n",
    "train_idhogar = train['idhogar']\n",
    "train_y = train['Target']\n",
    "train_X = train.drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "test_Id = test['Id']\n",
    "test_idhogar = test['idhogar']\n",
    "test_X = test.drop(['Id', 'idhogar'], axis=1)\n",
    "\n",
    "all_Id = pd.concat([train_Id, test_Id], axis=0, sort=False)\n",
    "all_idhogar = pd.concat([train_idhogar, test_idhogar], axis=0, sort=False)\n",
    "all_X = pd.concat([train_X, test_X], axis=0, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "54c83727",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:05.713369Z",
     "iopub.status.busy": "2023-02-07T15:04:05.711806Z",
     "iopub.status.idle": "2023-02-07T15:04:08.370098Z",
     "shell.execute_reply": "2023-02-07T15:04:08.369048Z"
    },
    "papermill": {
     "duration": 2.668404,
     "end_time": "2023-02-07T15:04:08.373886",
     "exception": false,
     "start_time": "2023-02-07T15:04:05.705482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier()"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix, f1_score, make_scorer\n",
    "import lightgbm as lgb\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(train_X, train_y, test_size=0.1, random_state=0)\n",
    "\n",
    "F1_scorer = make_scorer(f1_score, greater_is_better=True, average='macro')\n",
    "\n",
    "gbm = lgb.LGBMClassifier()\n",
    "\n",
    "gbm.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "938f9ac6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:08.387761Z",
     "iopub.status.busy": "2023-02-07T15:04:08.386854Z",
     "iopub.status.idle": "2023-02-07T15:04:08.436917Z",
     "shell.execute_reply": "2023-02-07T15:04:08.435929Z"
    },
    "papermill": {
     "duration": 0.059709,
     "end_time": "2023-02-07T15:04:08.439764",
     "exception": false,
     "start_time": "2023-02-07T15:04:08.380055",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(os.path.join(working_dir, '20180801_lgbm.pickle'), mode='wb') as f:\n",
    "    pickle.dump(gbm, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f482e894",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:08.453561Z",
     "iopub.status.busy": "2023-02-07T15:04:08.453142Z",
     "iopub.status.idle": "2023-02-07T15:04:08.777871Z",
     "shell.execute_reply": "2023-02-07T15:04:08.776619Z"
    },
    "papermill": {
     "duration": 0.334767,
     "end_time": "2023-02-07T15:04:08.781081",
     "exception": false,
     "start_time": "2023-02-07T15:04:08.446314",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pred = gbm.predict(test_X)\n",
    "pred = pd.Series(data=pred, index=test_Id.values, name='Target')\n",
    "pred = pd.concat([test_Id, pred], axis=1)\n",
    "submission = pred\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56267756",
   "metadata": {
    "papermill": {
     "duration": 0.005104,
     "end_time": "2023-02-07T15:04:08.791378",
     "exception": false,
     "start_time": "2023-02-07T15:04:08.786274",
     "status": "completed"
    },
    "tags": []
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a3742a03",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-07T15:04:08.804227Z",
     "iopub.status.busy": "2023-02-07T15:04:08.803446Z",
     "iopub.status.idle": "2023-02-07T15:04:08.825040Z",
     "shell.execute_reply": "2023-02-07T15:04:08.823916Z"
    },
    "papermill": {
     "duration": 0.031226,
     "end_time": "2023-02-07T15:04:08.827810",
     "exception": false,
     "start_time": "2023-02-07T15:04:08.796584",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix: \n",
      " [[  5   7   1   9]\n",
      " [  4  12   2  30]\n",
      " [  3   6   3  26]\n",
      " [  1   7   6 176]]\n",
      "macro F1 score: \n",
      " 0.38060490553529996\n"
     ]
    }
   ],
   "source": [
    "y_test_pred = gbm.predict(X_test)\n",
    "cm = confusion_matrix(y_test, y_test_pred)\n",
    "f1 = f1_score(y_test, y_test_pred, average='macro')\n",
    "print(\"confusion matrix: \\n\", cm)\n",
    "print(\"macro F1 score: \\n\", f1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 14.853966,
   "end_time": "2023-02-07T15:04:09.659677",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-02-07T15:03:54.805711",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
